package net.bwie.realtime.jtp.dwd.log.job1;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import net.bwie.realtime.jtp.dwd.log.function1.AppLogSplitProcessFunction;
import net.bwie.realtime.jtp.utils.KafkaUtil;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SideOutputDataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;

public class JtpAppLogEtlJob extends Exception{
    public static void main(String[] args) throws Exception {
        //创建执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        //数据源source
        DataStream<String> KfkStream = KafkaUtil.consumerKafka(env, "topic-log");
        KfkStream.print("Kafka");
        //数据转换transformation
        DataStream<String> pageStream = processLog(KfkStream);
        //数据输出到Kfk
        KafkaUtil.producerKafka(pageStream,"dwd-traffic-page-log");
        //触发执行
        env.execute("JtpAppLogEtlJob");
    }

    private static DataStream<String> processLog(DataStream<String> kfkStream) {
        //1.数据清洗
        DataStream<String> cleanStream = CleanData(kfkStream);
        //2.新老用户修改
        DataStream<String> isNewStream = IsNewUser(cleanStream);
        //3.数据分流
        DataStream<String> dataSplit = DataSplit(cleanStream);
        return isNewStream;
    }

    /**
     * 修改新老用户
     * @param cleanStream
     * @return
     */
    private static DataStream<String> DataSplit(DataStream<String> cleanStream) {
        OutputTag<String> errorLog = new OutputTag<String>("error-log") {
        };
        OutputTag<String> startLog = new OutputTag<String>("start-log") {
        };
        OutputTag<String> displayLog = new OutputTag<String>("display-log") {
        };
        OutputTag<String> actionLog = new OutputTag<String>("action-log") {
        };
        SingleOutputStreamOperator<String> splitStream = cleanStream.process(
                new AppLogSplitProcessFunction(errorLog, startLog, displayLog, actionLog)
        );
        SideOutputDataStream<String> errorStream = splitStream.getSideOutput(errorLog);
        KafkaUtil.producerKafka(errorStream,"error-log");
        SideOutputDataStream<String> startStream = splitStream.getSideOutput(startLog);
        KafkaUtil.producerKafka(startStream,"start-log");
        SideOutputDataStream<String> displayStream = splitStream.getSideOutput(displayLog);
        KafkaUtil.producerKafka(displayStream,"display-log");
        SideOutputDataStream<String> actionStream = splitStream.getSideOutput(actionLog);
        KafkaUtil.producerKafka(actionStream,"action-log");
        return splitStream;
    }


    private static DataStream<String> IsNewUser(DataStream<String> kfkStream) {
        //根据用户ID分组
        KeyedStream<String, String> midStream = kfkStream.keyBy(new KeySelector<String, String>() {
            @Override
            public String getKey(String s) throws Exception {
                return JSON.parseObject(s).getJSONObject("common").getString("mid");
            }
        });
        return midStream;
    }

    /**
     * 数据清洗
     * @param kfkStream
     * @return
     */
    private static DataStream<String> CleanData(DataStream<String> kfkStream) {
        final OutputTag<String> dirtyData = new OutputTag<String>("dirty-log"){};
        SingleOutputStreamOperator<String> dirtyStream = kfkStream.process(new ProcessFunction<String, String>() {
            @Override
            public void processElement(String value, Context ctx, Collector<String> out) throws Exception {
                try {
                    //解析JSON,没有异常正常输出
                    JSON.parseObject(value);
                    out.collect(value);
                } catch (Exception e) {
                    //捕获异常输出测流
                    ctx.output(dirtyData, value);
                }
            }
        });
        //读取测流数据消费到kfk
        SideOutputDataStream<String> dirtyLog = dirtyStream.getSideOutput(dirtyData);
        KafkaUtil.producerKafka(dirtyLog,"dirty-log");
        return dirtyStream;
    }
}
